
import pandas as pd
import numpy as np
import datetime as dt
import sys
import warnings
import IPython as ip
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import seaborn as sns
import scipy.stats as st
from scipy.stats import t, shapiro
from scipy.stats import normaltest
import statsmodels 
import statsmodels.api as sm 
import statsmodels.formula.api as smf
from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.graphics.gofplots import qqplot
# ACP
from sklearn.preprocessing import StandardScaler
from sklearn import decomposition
from sklearn.decomposition import PCA
from sklearn import decomposition
from sklearn import preprocessing


from IPython.display import display
import missingno as msno 
# Configuration pour travail avec fichier python "tools" de fonctions
%load_ext autoreload
%aimport tools
# Recharger les modules pour la conception des fichiers tools
%autoreload 1

# Set option
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)


warnings.filterwarnings("ignore")


# Import données
data = pd.read_csv('assets/datas/df_app_knnImputer.csv', sep='\t',parse_dates=[2,3], low_memory=False)


df = data.copy()
# Visualisation d'un échantillon de la population
df.sample(5)


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 240304 entries, 0 to 240303
Data columns (total 23 columns):
 #   Column                       Non-Null Count   Dtype         
---  ------                       --------------   -----         
 0   code                         240304 non-null  object        
 1   creator                      240304 non-null  object        
 2   created_datetime             240304 non-null  datetime64[ns]
 3   last_modified_datetime       240304 non-null  datetime64[ns]
 4   product_name                 240304 non-null  object        
 5   brands                       240304 non-null  object        
 6   categories_fr                240304 non-null  object        
 7   countries_fr                 240304 non-null  object        
 8   additives_n                  240304 non-null  float64       
 9   additives_fr                 240304 non-null  object        
 10  ingredients_from_palm_oil_n  240304 non-null  float64       
 11  nutrition_grade_fr           240304 non-null  object        
 12  main_category_fr             240304 non-null  object        
 13  energy_100g                  240304 non-null  float64       
 14  fat_100g                     240304 non-null  float64       
 15  saturated_fat_100g           240304 non-null  float64       
 16  carbohydrates_100g           240304 non-null  float64       
 17  sugars_100g                  240304 non-null  float64       
 18  fiber_100g                   240304 non-null  float64       
 19  proteins_100g                240304 non-null  float64       
 20  salt_100g                    240304 non-null  float64       
 21  sodium_100g                  240304 non-null  float64       
 22  nutrition_score_fr_100g      240304 non-null  float64       
dtypes: datetime64[ns](2), float64(12), object(9)
memory usage: 42.2+ MB


tools.get_description_variables(df,type_var='categ')


tools.get_description_variables(df,type_var='num')


add_per_year = df['code'].groupby(by=df['created_datetime'].dt.year).nunique()
modified_per_year = df['code'].groupby(by=df['last_modified_datetime'].dt.year).nunique()

fig=plt.figure(figsize=(12,8))

font_title = {'family': 'serif',
              'color':  '#114b98',
              'weight': 'bold',
              'size': 18,
             }

sns.set_style("whitegrid")
plt.plot(add_per_year, 
         color="#114b98", 
         label="Ajouts")
plt.plot(modified_per_year, 
         color="#00afe6", 
         label="Modifications")
plt.title("Evolution des créations et modifications de produits par année", 
          fontdict=font_title)
plt.xlabel("Année")
plt.ylabel("Nombre de produits")
plt.legend()
plt.savefig("assets/graphiques/Evolutions_dates.jpg")

plt.show()


float_columns  = df.select_dtypes(include=['float64']).columns.to_list()
object_columns = df.select_dtypes(include=['object']).columns.to_list()
datetime_columns = df.select_dtypes(include=['datetime64[ns]']).columns.to_list()


# On écarte les dates
# Variables numériques
cols_num = df.select_dtypes(include=[np.number]).columns.to_list()
# Variables quantitatives discrètes
cols_quant_discr = ['additives_n','ingredients_from_palm_oil_n','nutriscore_score_fr']
# Variables quantitatives continue
cols_quant_cont = ['energy_100g', 'fat_100g','saturated_fat_100g', 
                   'carbohydrates_100g', 'sugars_100g', 'fiber_100g',
                   'proteins_100g', 'salt_100g', 'sodium_100g']


# Mesures de tendances centrales des colonnes quantitatives continues
tools.stat_descriptives(df,cols_quant_cont)


def var_hist(var, i):
    subset = df[var]
    n_df_valide = len(df)
    xbar = np.mean(df[var]) # Moyenne
    sprime = np.std(df[var], ddof=1) # Ecart-type
    sprime2 = np.var(df[var], ddof=1) #Variance non biaisée
    
    ax = fig.add_subplot(i)
    ax.hist(subset, density=True)
    ax.axvline(xbar, color='r', linewidth=2, label="Moyenne empirique")
    bins = np.arange(df[var].min(),df[var].max(),0.05)
    y = st.norm.pdf(bins, xbar, sprime)
    ax.plot(bins, y, '--', label="Densité normale")
    ax.legend()
    ax.set_xlabel(var, fontsize=12)
    ax.set_ylabel('Densité', fontsize=12)
    ax.set_title('Distribution de '+str(var), fontsize=18)


liste_var = cols_quant_cont
plt.style.use('seaborn-whitegrid')
fig = plt.figure(figsize=(20,30),constrained_layout=False)
i = 331
for var in liste_var :
    var_hist(var, i)
    i+=1
plt.savefig("assets/graphiques/analyse univariee histo_dfComplet.jpg")


# Representation graphique des outliers:
a = 3  # nombre de lignes
b = 3  # nombre de colonnes
c = 1  # initialisation
fig = plt.figure(figsize=(20,8))

for i in df.loc[:, cols_quant_cont]: # pour toute les colonnnes quantatives
    plt.subplot(a, b, c) # maillage des subplot
    plt.title('{} (boxplot)'.format(i, a, b, c))# titres des box plot
    plt.xlabel(i) # xlabel = nom de la colonne
    sns.boxplot(x = df[i]) # faire un boxplot sns
    c = c + 1 # incrementation ==> création d'un nouveau box plot
plt.subplots_adjust(left=0.125, # gerer les espacements
                    bottom=0.1, 
                    right=0.9, 
                    top=0.9, 
                    wspace=0.2, 
                    hspace=0.35)


# On s'occupe ici uniquement des nutrigrades complétés
df_nutri = df[~(df['nutrition_grade_fr']=='0')]
# On s'occupe ici uniquement des nutrigrades complétés
df_nutriscore = df[~(df['nutrition_score_fr_100g']=='0')]
df_nutriscore = df_nutriscore[~(df_nutriscore['nutrition_grade_fr']=='0')]


# Courbe de distribution du nutriscore
plt.figure(figsize=(12, 8))

sns.histplot(df_nutriscore['nutrition_score_fr_100g'], kde=True,
             color='SteelBlue', label='Nutri_score pour 100g de produit')
plt.title("Distribution du nutri-score", fontsize=14)
plt.xlim(-15, 40)
plt.xlabel('Score', fontsize=12)
plt.ylabel('Nombre de produits par score', fontsize=12)
plt.legend()
plt.show()


fig = plt.figure(figsize=(15, 6))

ax1 = fig.add_subplot(1, 2, 1)
box = sns.boxplot(data=df_nutri['nutrition_score_fr_100g'], color='SteelBlue', ax=ax1)
# box.set(ylabel=unite)

plt.grid(False)

ax2 = fig.add_subplot(1, 2, 2)
ax2 = sm.qqplot(df_nutri['nutrition_score_fr_100g'],
             line='r', ax=ax2)
plt.grid(False)

fig.suptitle('Dispersion des nutrition-score-fr_100g', fontweight='bold', size=14)
plt.show()


col = ['nutrition_score_fr_100g']
tools.stat_descriptives(df_nutri,col)


# définition des bacs
# https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.IntervalIndex.from_tuples.html
liste_bins = pd.IntervalIndex.from_tuples(
    [(-15, -1), (0, 2), (3, 10), (10, 18), (19, 40)])
tools.distribution_variables_plages_perc_donnees(df_nutri,'nutrition_score_fr_100g',liste_bins)


# On visualise le nombre de valeurs uniques contenu dans les colonnes de type object
for col in df.select_dtypes('object'):
    print(f'{col:-<50} {df[col].nunique()}')

code---------------------------------------------- 240304
creator------------------------------------------- 2477
product_name-------------------------------------- 186889
brands-------------------------------------------- 46265
categories_fr------------------------------------- 16277
countries_fr-------------------------------------- 81
additives_fr-------------------------------------- 38480
nutrition_grade_fr-------------------------------- 6
main_category_fr---------------------------------- 2322


# Variables qualitatives ou modalités
# Variables qualitatives nominales
cols_qual_nom = ['code','creator','product_name','brands', 
                 'categories_fr','main_category_fr', 'countries_fr','additives_fr']


def top_N_pie (df,var,name,n,taille,perc) : 
    ''' 
    Fonction qui visualise les n plus grand d'une colonne avec ou sans pourcentage
    parametres : 
        df
        var : colonne ciblée
        name : 'nom de la colonne '
        n : nombre de top voulu
        taille : taille du pieplot
        per : si True : affiche les pourcentages
    '''
    target = df.groupby(by=var)['code'].nunique().sort_values(ascending=False)
    # Graphiques top N
    
    fig, ax = plt.subplots(figsize=(taille, taille), subplot_kw=dict(aspect="equal"))
    explodes = np.zeros(n)
    explodes[0] = .1
    # calcul des pourcentages
    if perc: 
        def pct_tot(pct):
            tot = round(pct*target[:n].sum(),0)
            tot_pct = tot/target.sum()
            return "{:.1f}%\n({:.0f})".format(tot_pct,(tot/100))
        plt.pie(target[:n], labels=target[:n].index,
            startangle=45,
            shadow=True,
            autopct=lambda pct: pct_tot(pct),
            explode=explodes,
            textprops=dict(color="black",size=12, weight="bold"))
    else : 
        plt.pie(target[:n], labels=target[:n].index,
                startangle=45,
                shadow=True,
                explode=explodes,
                textprops=dict(color="black",size=10, weight="bold"))
    plt.title(f"TOP {n} : {name}",fontweight='bold',fontsize=24)
    plt.show()


# Nombre de créateurs, sources des données
print(f"Nombre de sources unique : {df['creator'].nunique()}")

Nombre de sources unique : 2477


top_N_pie(df,'creator','Contributeurs',5,12,True)
plt.savefig("assets/graphiques/Top_Contributeurs.jpg")

<Figure size 640x480 with 0 Axes>


df['brands'].nunique()

46265


# On s'occupe ici uniquement des catégories renseignées
df_brands = df[~(df['brands']=='inconnue')]
df_brands.shape

(237159, 23)


# Tableau fréquences
dico = df_brands.groupby('brands')['brands'].count().sort_values(ascending=False).to_dict()
nom = 'brands'
col1 = 'Nom_' + nom
col2 = 'Nbr_' + nom
col3 = 'Fréquence (%)'
df_gpe = pd.DataFrame(dico.items(), columns=[col1, col2])
df_gpe[col3] = (df_gpe[col2] * 100) / len(df_brands)
df_gpe.head(10)


# Wordecloud
from wordcloud import WordCloud

wordcloud = WordCloud(width=800,height=400, background_color="white",max_words=100).generate_from_frequencies(dico)
plt.figure(figsize=(12, 10))
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis("off")
plt.show()


 df_gp_red = df_gpe.head(10)
sns.set_style("whitegrid")
plt.figure(figsize=(8, 4))
sns.barplot(
    y=df_gp_red[col1],
    x=df_gp_red[col3],
    data=df_gp_red,
    color='SteelBlue')
plt.title('Répartition de la présence des marques dans le jeu de données')
plt.grid(False)
plt.tight_layout()
plt.show()


df['categories_fr'].nunique()

16277


# On s'occupe ici uniquement des catégories renseignées
df_categ = df[~(df['categories_fr']=='inconnu')]


tools.affiche_wordcloud_tabfreq(df_categ,'categories_fr','categories',)


tools.affiche_wordcloud_tabfreq(df_categ,'main_category_fr','Main categories',affword=False)


# On s'occupe ici uniquement des catégories renseignées
df_additives = df[~(df['additives_fr']==' ')]


df_additives_target = df_additives.copy()
df_additives_target = df_additives_target[df_additives_target['additives_fr'].str.contains("338|339|340|341|343|450|451|452")]


tools.affiche_wordcloud_tabfreq(df_additives_target,'additives_fr','Additives',affword=False)


# Variables qualitatives ordinales
cols_qual_ord = ['nutriscore_grade_fr']


# On s'occupe ici uniquement des nutrigrades complétés
df_nutri = df[~(df['nutrition_grade_fr']=='0')]


nutrition_grade = df_nutri.groupby(by='nutrition_grade_fr')['code'].nunique().sort_values(ascending=False)

fig, ax = plt.subplots(figsize=(6, 6), subplot_kw=dict(aspect="equal"))
explodes = np.zeros(5)
explodes[0] = .1

plt.pie(nutrition_grade, labels=nutrition_grade.index, 
        startangle=0, 
        colors=['#ee8100','#fecb02','#e63e11','#038141','#85bb2f'],
        shadow=True,
        explode=explodes,
        autopct='%1.1f%%',
        textprops=dict(color="black",size=12, weight="bold"))
plt.title("Répartition des Nutrition_grade", fontdict=font_title)
plt.savefig("assets/graphiques/Répartion_nutrigrdes.jpg")
plt.show()


sns.pairplot(df_nutri.sample(frac=0.05), hue="nutrition_grade_fr")
plt.savefig("assets/graphiques/Pairplot_Nutrition grade.jpg")


sns.clustermap(df_nutri.corr(),annot=True)

<seaborn.matrix.ClusterGrid at 0x17764f7f3a0>


# Répartition des variables quantitative en fonction du nutrigrade
colors_nutri = ['#038141','#85bb2f','#fecb02','#ee8100','#e63e11']
fig = plt.figure(figsize=(20, 35))

for i, c in enumerate(df_nutriscore.select_dtypes('float'), 1):
    ax = fig.add_subplot(6, 2, i)
    sns.boxplot(data=df_nutriscore, x='nutrition_grade_fr', y=c,order='abcde', ax=ax,palette=colors_nutri)
    plt.grid(False)
    plt.xticks(fontsize=16)
    plt.yticks(fontsize=16)

plt.tight_layout(rect=[0, 0.03, 1, 0.95])
plt.suptitle('Répartition des variables quantitatives en fonction du nutrigrade', fontsize=30)
plt.savefig("assets/graphiques/Répartition des variables quantitatives en fonction du nutrigrade.jpg")
plt.show()


df_test = df_nutriscore.copy()
# on ajoute une colonne corespondant à la labelEncoder de la variable nutrigrade
le = preprocessing.LabelEncoder()
le.fit(["a", "b", "c", "d","e"])
df_test['nutrition_grade_fr_le'] = le.transform(df_test['nutrition_grade_fr'])


# Test de Shapiro-Wilk
shapiro(df_nutriscore['proteins_100g'])

ShapiroResult(statistic=0.8269742131233215, pvalue=0.0)


def test_normalite(df,var):
    ''' Création d'une fonction pour tester la normalioté de la distribution des variables
    '''
    print(f"Variable : {var}")
    stat, p = shapiro(df[var]) 
    print ('Test Shapiro')
    print('stat=%.3f, p=%.3f' % (stat,p))
    #interprétation
    if p > 0.05:
        print("Distribution probablement gaussienne")
    else:
        print("Distribution non Gaussienne")
    print("------------------------------------------")
    print("Test normaltest (d'Agostino)")
    stat, p = normaltest(df[var])
    print('Statistics=%.3f, p=%.3f' % (stat, p))
# interpretation
    alpha = 0.05
    if p < alpha:  # null hypothesis: x comes from a normal distribution
        print("H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) ")
    else:
        print("H0 ne peut être rejetée - la distribution des données suit une loi normale (P>0,05)")
    print("________________________________________________________")


# On s'occupe ici uniquement des nutrigrades et nutriscore complétés
df_nutriscore = df[~(df['nutrition_score_fr_100g']=='0')]
df_nutriscore = df_nutriscore[~(df_nutriscore['nutrition_grade_fr']=='0')]


# graph
sns.histplot(data=df_nutriscore.sort_values("nutrition_grade_fr"), x="nutrition_score_fr_100g", hue="nutrition_grade_fr")
plt.show()


fig, axes = plt.subplots(1, 2, sharex=False, sharey=False, figsize=(21,8))
fig.suptitle(r"Répartition des scores Nutriscore et de leurs grades" "\n", fontsize=22)

sns.histplot(data=df_nutriscore.sort_values("nutrition_grade_fr"), x="nutrition_grade_fr", hue="nutrition_grade_fr", ax=axes[0])
axes[0].set_title('Grades de Nutriscores')
axes[0].set_xlabel("nutrition_grade_fr")
axes[0].set_ylabel("Nombre de produits")

sns.histplot(data=df_nutriscore.sort_values("nutrition_grade_fr"), x="nutrition_score_fr_100g", hue="nutrition_grade_fr", ax=axes[1])
axes[1].set_title('Scores de Nutriscores')
axes[1].set_xlabel("Score Nutriscore")
axes[1].set_ylabel("Nombre de produits")

plt.show()


# Préparation des variables de travail pour les graphiques et les tests
gb = df_nutriscore.groupby('nutrition_grade_fr')['nutrition_score_fr_100g']
df_nutriscore_nutrigrade = pd.DataFrame([gb.get_group(n).values for n in list('abcde')],
                             index=list('abcde')).T


tools.stat_descriptives(df_nutriscore_nutrigrade, ['a', 'b', 'c', 'd', 'e'])


test_normalite(df_test,'nutrition_score_fr_100g')

Variable : nutrition_score_fr_100g
Test Shapiro
stat=0.968, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=51833.900, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________


# Préparation des variables de travail pour les graphiques et les tests
gp = df_nutriscore.groupby('nutrition_grade_fr')['proteins_100g']
df_nutrigrade_protein = pd.DataFrame([gp.get_group(n).values for n in list('abcde')],
                             index=list('abcde')).T
df_nutrigrade_protein = df_nutrigrade_protein.dropna()
df_a = df_nutrigrade_protein['a']
df_b = df_nutrigrade_protein['b']
df_c = df_nutrigrade_protein['c']
df_d = df_nutrigrade_protein['d']
df_e = df_nutrigrade_protein['e']


plt.figure(figsize=[10, 10])
# colors_nutri = ['#038141','#85bb2f','#fecb02','#ee8100','#e63e11']
# Boxplot protéines/nutri-score grade
plt.subplot(2, 1, 1)
sns.boxplot(data=df_nutriscore, x='nutrition_grade_fr', y='proteins_100g',
            palette=colors_nutri, order='abcde')
plt.ylim(0, 100)
plt.ylabel('Nombre de g de protéines pour 100g de produit', fontsize=12)
plt.xlabel('Nutri-grade_fr', fontsize=12)
plt.title('Protéines par nutri-grade', fontsize=14)
plt.grid(False)
# Ajout moyenne des protéines pour tous les produits
moyenne_proteines = df_nutriscore['proteins_100g'].mean()
plt.axhline(y=moyenne_proteines, color='r')

# Violinplot protéines/nutri-score grade
plt.subplot(2, 1, 2)
sns.violinplot(data=df_nutriscore, x='nutrition_grade_fr', y='proteins_100g',
               palette=colors_nutri, order='abcde')
plt.ylabel('Nombre de g de protéines pour 100g de produit', fontsize=12)
plt.xlabel('Nutri-grade', fontsize=12)
plt.grid(False)
# Ajout moyenne des protéines pour tous les produits
plt.axhline(y=moyenne_proteines, color='r')

plt.show()


# Distplot protéines
sns.distplot(df_nutriscore['proteins_100g'], bins=100, color='SteelBlue')
plt.grid(False)


qqplot(df_nutriscore['proteins_100g'], line='r')
plt.grid(False)
plt.show()


test_normalite(df_nutriscore,'proteins_100g')

Variable : proteins_100g
Test Shapiro
stat=0.827, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=90840.247, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________


# Statistiques descriptives
tools.stat_descriptives(df_nutriscore, ['proteins_100g'])


import matplotlib.patches as mpatches

fig = plt.figure(figsize=(8, 6))
label_patches = []

sns.kdeplot(df_nutriscore['proteins_100g'], color='Blue')
label_patch = mpatches.Patch(
    color='Blue',
    label='Ensemble des nutrigrades')
label_patches.append(label_patch)
plt.grid(False)
plt.xlim([-1, 40])

i = 1
for n, c in zip(list('abcde'), colors_nutri):
    i += 1
    sns.kdeplot(df_nutrigrade[n], color=c)
    label_patch = mpatches.Patch(color=c, label=n)
    label_patches.append(label_patch)
    plt.grid(False)
    plt.xlim([-1, 40])

fig.suptitle('Distribution des protéines', fontweight='bold', fontsize=14)
plt.legend(handles=label_patches,bbox_to_anchor=(1.05,1),loc=2,borderaxespad=0., facecolor='white')
plt.tight_layout()
plt.grid(False)
plt.show()


# Histogramme des protéines, général et par nutri-score
def graph_distribution(df,var):
    fig = plt.figure(figsize=(8, 15))

    fig.add_subplot(6, 1, 1)
    sns.distplot(df[var], bins=70, color='SteelBlue')
    plt.grid(False)

    i = 1
    for n, c in zip(list('abcde'), colors_nutri):
        i += 1
        ax = fig.add_subplot(6, 1, i)
        sns.distplot(df_nutrigrade[n], color=c, bins=70)
        plt.grid(False)
    fig.suptitle(f'Histogramme des {var}',
                 fontweight='bold', fontsize=14)

    plt.tight_layout(rect=[0, 0.0, 1, 0.93])
    plt.grid(False)
    plt.show()


graph_distribution(df_nutriscore,'proteins_100g')


# On test la normalité de la distribution sur les proteines par nutrigrade
for col in df_nutrigrade.columns:
    print(f"Nutrigrade : {col}")
    test_normalite(df_nutrigrade_protein,col)

Nutrigrade : a
Variable : a
Test Shapiro
stat=0.855, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=16449.014, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________
Nutrigrade : b
Variable : b
Test Shapiro
stat=0.722, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=23643.529, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________
Nutrigrade : c
Variable : c
Test Shapiro
stat=0.828, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=14416.667, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________
Nutrigrade : d
Variable : d
Test Shapiro
stat=0.847, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=10600.846, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________
Nutrigrade : e
Variable : e
Test Shapiro
stat=0.829, p=0.000
Distribution non Gaussienne
------------------------------------------
Test normaltest (d'Agostino)
Statistics=9263.825, p=0.000
H0 peut être rejetée - H1 : la distribution des données ne suit pas la loi normale (P<0,05) 
________________________________________________________


liste_bins = pd.IntervalIndex.from_tuples(
    [(0, 4), (5, 9), (10, 14), (15, 19), (20, 24), (25, 29),
     (30, 34), (35, 39), (40, 100)])
tools.distribution_variables_plages_perc_donnees(df_nutriscore, 'proteins_100g', liste_bins)


df_nutriscore['proteins_100g'].describe()

count    203405.000000
mean          7.782852
std           8.036600
min           0.000000
25%           2.000000
50%           5.710000
75%          10.710000
max         100.000000
Name: proteins_100g, dtype: float64


df_nutrigrade_protein.describe()


# Anova OLS
anova_nutrigrade = smf.ols('proteins_100g~nutrition_grade_fr', data=df_nutriscore).fit()
print(anova_nutrigrade.summary())

                            OLS Regression Results                            
==============================================================================
Dep. Variable:          proteins_100g   R-squared:                       0.028
Model:                            OLS   Adj. R-squared:                  0.028
Method:                 Least Squares   F-statistic:                     1491.
Date:                Fri, 13 Jan 2023   Prob (F-statistic):               0.00
Time:                        18:31:24   Log-Likelihood:            -7.0958e+05
No. Observations:              203405   AIC:                         1.419e+06
Df Residuals:                  203400   BIC:                         1.419e+06
Df Model:                           4                                         
Covariance Type:            nonrobust                                         
===========================================================================================
                              coef    std err          t      P>|t|      [0.025      0.975]
-------------------------------------------------------------------------------------------
Intercept                   8.3360      0.044    191.083      0.000       8.250       8.421
nutrition_grade_fr[T.b]    -3.2269      0.062    -51.651      0.000      -3.349      -3.104
nutrition_grade_fr[T.c]    -1.3744      0.058    -23.593      0.000      -1.489      -1.260
nutrition_grade_fr[T.d]     0.2438      0.055      4.457      0.000       0.137       0.351
nutrition_grade_fr[T.e]     0.8275      0.059     13.988      0.000       0.712       0.943
==============================================================================
Omnibus:                    91895.107   Durbin-Watson:                   0.937
Prob(Omnibus):                  0.000   Jarque-Bera (JB):           713257.449
Skew:                           2.014   Prob(JB):                         0.00
Kurtosis:                      11.242   Cond. No.                         6.46
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.


import matplotlib.patches as mpatches

fig = plt.figure(figsize=(8, 6))
label_patches = []

sns.kdeplot(df_nutriscore['sodium_100g'], color='Blue')
label_patch = mpatches.Patch(
    color='Blue',
    label='Ensemble des nutrigrades')
label_patches.append(label_patch)
plt.grid(False)
plt.xlim([-1, 40])

i = 1
for n, c in zip(list('abcde'), colors_nutri):
    i += 1
    sns.kdeplot(df_nutrigrade[n], color=c)
    label_patch = mpatches.Patch(color=c, label=n)
    label_patches.append(label_patch)
    plt.grid(False)
    plt.xlim([-1, 40])

fig.suptitle('Distribution sodium', fontweight='bold', fontsize=14)
plt.legend(handles=label_patches,bbox_to_anchor=(1.05,1),loc=2,borderaxespad=0., facecolor='white')
plt.tight_layout()
plt.grid(False)
plt.show()


df_nutriscore['sodium_100g'].describe()

count    203405.000000
mean          0.490369
std           1.556126
min           0.000000
25%           0.039370
50%           0.256000
75%           0.536000
max          39.370079
Name: sodium_100g, dtype: float64


liste_bins = pd.IntervalIndex.from_tuples(
    [(0, 0.2), (0.3, 0.5), (0.5, 0.7), (0.8, 1), (1.1, 1.5), (1.6, 2.5),
     (2.6, 3), (3, 4), (4, 40)])
tools.distribution_variables_plages_perc_donnees(df_nutriscore, 'sodium_100g', liste_bins)


graph_distribution(df_nutriscore,'sodium_100g')


# Préparation des variables de travail pour les graphiques et les tests
gb = df_nutriscore.groupby('nutrition_grade_fr')['sodium_100g']
df_nutrigrade_sodium = pd.DataFrame([gb.get_group(n).values for n in list('abcde')],
                             index=list('abcde')).T


df_nutrigrade_sodium.describe()


# Anova OLS
anova_nutrigrade = smf.ols('sodium_100g~nutrition_grade_fr', data=df_nutriscore).fit()
print(anova_nutrigrade.summary())

                            OLS Regression Results                            
==============================================================================
Dep. Variable:            sodium_100g   R-squared:                       0.020
Model:                            OLS   Adj. R-squared:                  0.020
Method:                 Least Squares   F-statistic:                     1056.
Date:                Fri, 13 Jan 2023   Prob (F-statistic):               0.00
Time:                        18:28:12   Log-Likelihood:            -3.7647e+05
No. Observations:              203405   AIC:                         7.530e+05
Df Residuals:                  203400   BIC:                         7.530e+05
Df Model:                           4                                         
Covariance Type:            nonrobust                                         
===========================================================================================
                              coef    std err          t      P>|t|      [0.025      0.975]
-------------------------------------------------------------------------------------------
Intercept                   0.1305      0.008     15.383      0.000       0.114       0.147
nutrition_grade_fr[T.b]     0.0767      0.012      6.317      0.000       0.053       0.101
nutrition_grade_fr[T.c]     0.5417      0.011     47.820      0.000       0.519       0.564
nutrition_grade_fr[T.d]     0.5174      0.011     48.651      0.000       0.497       0.538
nutrition_grade_fr[T.e]     0.4620      0.012     40.169      0.000       0.439       0.485
==============================================================================
Omnibus:                   390895.401   Durbin-Watson:                   1.250
Prob(Omnibus):                  0.000   Jarque-Bera (JB):        753156329.427
Skew:                          15.237   Prob(JB):                         0.00
Kurtosis:                     299.542   Cond. No.                         6.46
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.


df_test.columns

Index(['code', 'creator', 'created_datetime', 'last_modified_datetime',
       'product_name', 'brands', 'categories_fr', 'countries_fr',
       'additives_n', 'additives_fr', 'ingredients_from_palm_oil_n',
       'nutrition_grade_fr', 'main_category_fr', 'energy_100g', 'fat_100g',
       'saturated_fat_100g', 'carbohydrates_100g', 'sugars_100g', 'fiber_100g',
       'proteins_100g', 'salt_100g', 'sodium_100g', 'nutrition_score_fr_100g',
       'nutrition_grade_fr_le'],
      dtype='object')


def backward_selected(data, response):
    """Linear model designed by backward selection.

    Parameters:
    -----------
    data : pandas DataFrame with all possible predictors and response

    response: string, name of response column in data

    Returns:
    --------
    model: an "optimal" fitted statsmodels linear model
           with an intercept
           selected by backward selection
           evaluated by parameters p-value
    """
    remaining = set(data._get_numeric_data().columns)
    if response in remaining:
        remaining.remove(response)
    cond = True

    while remaining and cond:
        formula = "{} ~ {} + 1".format(response,' + '.join(remaining))
        print('_______________________________')
        print(formula)
        model = smf.ols(formula, data).fit()
        score = model.pvalues[1:]
        toRemove = score[score == score.max()]
        if toRemove.values > 0.05:
            print('remove', toRemove.index[0], '(p-value :', round(toRemove.values[0],3), ')')
            remaining.remove(toRemove.index[0])
        else:
            cond = False
            print('is the final model !')
        print('')
    print(model.summary())
    
    return model


# Intanciation de la régression linéaire multiple
colonnes = ['nutrition_grade_fr_le','nutrition_score_fr_100g','main_category_fr', 'energy_100g', 'fat_100g',
       'saturated_fat_100g', 'carbohydrates_100g', 'sugars_100g', 'fiber_100g',
       'proteins_100g', 'sodium_100g','salt_100g']
reg_backward = backward_selected(df_test[colonnes], 'nutrition_grade_fr_le')

_______________________________
nutrition_grade_fr_le ~ fat_100g + carbohydrates_100g + sodium_100g + sugars_100g + proteins_100g + fiber_100g + nutrition_score_fr_100g + salt_100g + energy_100g + saturated_fat_100g + 1
remove salt_100g (p-value : 0.209 )

_______________________________
nutrition_grade_fr_le ~ fat_100g + carbohydrates_100g + sodium_100g + sugars_100g + proteins_100g + fiber_100g + nutrition_score_fr_100g + energy_100g + saturated_fat_100g + 1
is the final model !

                              OLS Regression Results                             
=================================================================================
Dep. Variable:     nutrition_grade_fr_le   R-squared:                       0.920
Model:                               OLS   Adj. R-squared:                  0.920
Method:                    Least Squares   F-statistic:                 2.589e+05
Date:                   Fri, 13 Jan 2023   Prob (F-statistic):               0.00
Time:                           15:40:29   Log-Likelihood:                -93132.
No. Observations:                 203405   AIC:                         1.863e+05
Df Residuals:                     203395   BIC:                         1.864e+05
Df Model:                              9                                         
Covariance Type:               nonrobust                                         
===========================================================================================
                              coef    std err          t      P>|t|      [0.025      0.975]
-------------------------------------------------------------------------------------------
Intercept                   0.9686      0.002    548.712      0.000       0.965       0.972
fat_100g                    0.0005      0.000      4.674      0.000       0.000       0.001
carbohydrates_100g         -0.0008   5.85e-05    -13.802      0.000      -0.001      -0.001
sodium_100g                -0.0182      0.001    -31.608      0.000      -0.019      -0.017
sugars_100g                -0.0021   6.64e-05    -31.534      0.000      -0.002      -0.002
proteins_100g              -0.0020      0.000    -15.832      0.000      -0.002      -0.002
fiber_100g                  0.0026      0.000     10.933      0.000       0.002       0.003
nutrition_score_fr_100g     0.1549      0.000    895.651      0.000       0.155       0.155
energy_100g             -5.347e-05   3.08e-06    -17.341      0.000   -5.95e-05   -4.74e-05
saturated_fat_100g         -0.0121      0.000    -69.869      0.000      -0.012      -0.012
==============================================================================
Omnibus:                     1146.143   Durbin-Watson:                   1.527
Prob(Omnibus):                  0.000   Jarque-Bera (JB):             1613.150
Skew:                          -0.056   Prob(JB):                         0.00
Kurtosis:                       3.421   Cond. No.                     2.92e+03
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 2.92e+03. This might indicate that there are
strong multicollinearity or other numerical problems.


# On cherche l'élément qui explique le mieux le nutrigrade
anova_nutrigrade = smf.ols('energy_100g~nutrition_score_fr_100g', data=df_test).fit()
print(anova_nutrigrade.summary())

                            OLS Regression Results                            
==============================================================================
Dep. Variable:            energy_100g   R-squared:                       0.400
Model:                            OLS   Adj. R-squared:                  0.400
Method:                 Least Squares   F-statistic:                 1.356e+05
Date:                Fri, 13 Jan 2023   Prob (F-statistic):               0.00
Time:                        15:40:29   Log-Likelihood:            -1.5846e+06
No. Observations:              203405   AIC:                         3.169e+06
Df Residuals:                  203403   BIC:                         3.169e+06
Df Model:                           1                                         
Covariance Type:            nonrobust                                         
===========================================================================================
                              coef    std err          t      P>|t|      [0.025      0.975]
-------------------------------------------------------------------------------------------
Intercept                 694.5013      1.842    377.061      0.000     690.891     698.111
nutrition_score_fr_100g    52.7403      0.143    368.227      0.000      52.460      53.021
==============================================================================
Omnibus:                    14505.600   Durbin-Watson:                   0.896
Prob(Omnibus):                  0.000   Jarque-Bera (JB):            17898.063
Skew:                           0.692   Prob(JB):                         0.00
Kurtosis:                       3.445   Cond. No.                         18.3
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.


# On cherche l'élément qui explique le mieux le nutrigrade
anova_nutrigrade = smf.ols('nutrition_grade_fr_le~nutrition_score_fr_100g', data=df_test).fit()
print(anova_nutrigrade.summary())

                              OLS Regression Results                             
=================================================================================
Dep. Variable:     nutrition_grade_fr_le   R-squared:                       0.916
Model:                               OLS   Adj. R-squared:                  0.916
Method:                    Least Squares   F-statistic:                 2.211e+06
Date:                   Fri, 13 Jan 2023   Prob (F-statistic):               0.00
Time:                           15:40:29   Log-Likelihood:                -98032.
No. Observations:                 203405   AIC:                         1.961e+05
Df Residuals:                     203403   BIC:                         1.961e+05
Df Model:                              1                                         
Covariance Type:               nonrobust                                         
===========================================================================================
                              coef    std err          t      P>|t|      [0.025      0.975]
-------------------------------------------------------------------------------------------
Intercept                   0.8888      0.001    720.419      0.000       0.886       0.891
nutrition_score_fr_100g     0.1427   9.59e-05   1487.052      0.000       0.142       0.143
==============================================================================
Omnibus:                     2100.768   Durbin-Watson:                   1.490
Prob(Omnibus):                  0.000   Jarque-Bera (JB):             3539.915
Skew:                           0.043   Prob(JB):                         0.00
Kurtosis:                       3.641   Cond. No.                         18.3
==============================================================================

Notes:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.


df_acp = df_nutriscore.copy()


df_acp.columns

Index(['code', 'creator', 'created_datetime', 'last_modified_datetime',
       'product_name', 'brands', 'categories_fr', 'countries_fr',
       'additives_n', 'additives_fr', 'ingredients_from_palm_oil_n',
       'nutrition_grade_fr', 'main_category_fr', 'energy_100g', 'fat_100g',
       'saturated_fat_100g', 'carbohydrates_100g', 'sugars_100g', 'fiber_100g',
       'proteins_100g', 'salt_100g', 'sodium_100g', 'nutrition_score_fr_100g'],
      dtype='object')


# On isole les variables de notre ACP
cols_acp = ['energy_100g','fat_100g', 'saturated_fat_100g', 'carbohydrates_100g', 'sugars_100g',
           'fiber_100g', 'proteins_100g', 'salt_100g', 'sodium_100g','nutrition_score_fr_100g']


#Centrage et réduction
X = df_acp[cols_acp]
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)

#Instanciation de l'ACP
pca = PCA(svd_solver='full').fit(X_scaled)
X_projected = pca.transform(X_scaled)


#Variances expliquées
varexpl = pca.explained_variance_ratio_*100

#Projection de l'éboulis des valeurs propres
taux_var_exp = pca.explained_variance_ratio_
scree = taux_var_exp * 100
plt.bar(np.arange(len(scree)) + 1, scree, color='SteelBlue')
ax1 = plt.gca()
ax2 = ax1.twinx()
ax2.plot(np.arange(len(scree)) + 1, scree.cumsum(), c='red', marker='o')
ax2.set_ylabel('Taux cumulatif de l\'inertie')
ax1.set_xlabel('Rang de l\'axe d\'inertie')
ax1.set_ylabel('Pourcentage d\'inertie')
for i, p in enumerate(ax1.patches):
    ax1.text(p.get_width() /
             5 +
             p.get_x(),
             p.get_height() +
             p.get_y() +
             0.3,
             '{:.0f}%'.format(taux_var_exp[i] *100),
             fontsize=8,color='k')
plt.title("Eboulis des valeurs propres", fontdict=font_title)
plt.gcf().set_size_inches(8, 4)
plt.grid(False)
plt.show(block=False)


print("Le premier plan factoriel couvre une inertie de {:.2f}% et le second plan : {:.2f}%.".format(varexpl[0:2].sum(),
                                                                                                     varexpl[0:4].sum()))

Le premier plan factoriel couvre une inertie de 52.53% et le second plan : 83.15%.


#Espace des composantes principales
pcs = pca.components_

#Matrice des corrélations variables x facteurs
p = X.shape[1]
sqrt_valprop = np.sqrt(pca.explained_variance_)
corvar = np.zeros((p, p))
for dim in range(p):
    corvar[:,dim] = pcs[dim,:] * sqrt_valprop[dim]

#on affiche pour les deux premiers plans factoriels 
corr_matrix = pd.DataFrame({'feature':X.columns,'CORR_F1':corvar[:,0],'CORR_F2':corvar[:,1], 
              'CORR_F3':corvar[:,2], 'CORR_F4':corvar[:,3]})
corr_matrix


#Variable Illustrative
ivNutrigrade = df_acp['nutrition_grade_fr'].values

#Encodage des grades pour l'acp
from sklearn.preprocessing import LabelEncoder
encoder = LabelEncoder()
ivNutrigrade = encoder.fit_transform(ivNutrigrade)
ivNutrigrade = ivNutrigrade.reshape((ivNutrigrade.shape[0],1))

#Corrélation de la variable illustrative avec les axes factoriels 
corrIv = np.zeros((ivNutrigrade.shape[1],p))
for j in range(p): 
    for k in range(ivNutrigrade.shape[1]): 
        corrIv[k,j] = np.corrcoef(ivNutrigrade[:,k],X_projected[:,j])[0,1]


def cerle_corr(pcs, n_comp, pca, axis_ranks, 
               labels=None, label_rotation=0, 
               illustrative_var_label=None, illustrative_var_corr=None):
    for d1, d2 in axis_ranks:
        if d2 < n_comp:
            
            # initialisation de la figure
            fig=plt.figure(figsize=(10,10))
            fig.subplots_adjust(left=0.1,right=0.9,bottom=0.1,top=0.9)
            ax=fig.add_subplot(111)
            ax.set_aspect('equal', adjustable='box') 
            ax.set_facecolor("ivory")

            #détermination des limites du graphique
            ax.set_xlim(-1,1) 
            ax.set_ylim(-1,1) 

            #affichage des flèches 
            plt.quiver(np.zeros(pcs.shape[1]), np.zeros(pcs.shape[1]),
                       pcs[d1,:],pcs[d2,:], 
                       angles='xy', scale_units='xy', scale=1, 
                       color="black", alpha=0.5)
            # et noms de variables
            for i,(x,y) in enumerate(pcs[[d1,d2]].T):
                plt.annotate(labels[i],(x,y),
                             ha='center', va='center',
                             fontsize='14',color="black", alpha=0.8) 

            #variable illustrative
            if illustrative_var_label is not None :
                plt.annotate(illustrative_var_label,
                             (illustrative_var_corr[0,d1],illustrative_var_corr[0,d2]),
                             color='b')
                plt.quiver(np.zeros(pcs.shape[1]), np.zeros(pcs.shape[1]),
                                   illustrative_var_corr[0,d1],illustrative_var_corr[0,d2], 
                                   angles='xy', scale_units='xy', scale=1, color="b", alpha=0.5)

            #ajouter les axes 
            plt.plot([-1,1],[0,0],linewidth=1, color='black', ls='--') 
            plt.plot([0,0],[-1,1],linewidth=1, color='black', ls='--')

            #ajouter un cercle 
            cercle = plt.Circle((0,0),1,color='steelblue',fill=False) 
            ax.add_artist(cercle) 

            # nom des axes, avec le pourcentage d'inertie expliqué
            plt.xlabel('F{} ({}%)'.format(d1+1, round(100*pca.explained_variance_ratio_[d1],1)))
            plt.ylabel('F{} ({}%)'.format(d2+1, round(100*pca.explained_variance_ratio_[d2],1)))

            plt.title("Cercle des corrélations (F{} et F{})".format(d1+1, d2+1), fontdict=font_title)
            plt.show(block=False)


cerle_corr(pcs, 4, pca, [(0,1),(2,3)], labels = np.array(X.columns), 
           illustrative_var_label="Nutriscore_grade", illustrative_var_corr = corrIv)


def plot_plans_factoriels_nutrigrade(X_projected, n_comp, pca, axis_ranks, labels=None, alpha=1, illustrative_var=None):
    for d1,d2 in axis_ranks:
        if d2 < n_comp:
            # initialisation de la figure       
            fig = plt.figure(figsize=(12,8))
        
            # affichage des points
            if illustrative_var is None:
                plt.scatter(X_projected[:, d1], X_projected[:, d2], alpha=alpha)
            else:
                illustrative_var = np.array(illustrative_var)
                label_patches = []
                colors = ['#038141', '#85bb2f', '#fecb02', '#ee8100', '#e63e11']
                i = 0
                for value in np.unique(illustrative_var):
                    selected = np.where(illustrative_var == value)
                    plt.scatter(X_projected[selected, d1], X_projected[selected, d2], alpha=alpha, label=value, c=colors[i])
                    label_patch = mpatches.Patch(color=colors[i],
                                                 label=value)
                    label_patches.append(label_patch)
                    i += 1
                    plt.legend(
                        handles=label_patches,
                        bbox_to_anchor=(1.05,1),loc=2,borderaxespad=0.,facecolor='white')
               # plt.legend()
                

            # affichage des labels des points
            if labels is not None:
                for i,(x,y) in enumerate(X_projected[:,[d1,d2]]):
                    plt.text(x, y, labels[i],
                              fontsize='14', ha='center',va='center') 
                    
            
            # détermination des limites du graphique
            boundary = np.max(np.abs(X_projected[:, [d1,d2]]))*1.1
            plt.xlim([-boundary,boundary])
            plt.ylim([-boundary,boundary])
        
            # affichage des lignes horizontales et verticales
            plt.plot([-100, 100], [0, 0], color='grey', ls='--')
            plt.plot([0, 0], [-100, 100], color='grey', ls='--')

            # nom des axes, avec le pourcentage d'inertie expliqué
            plt.xlabel('F{} ({}%)'.format(d1+1, round(100*pca.explained_variance_ratio_[d1],1)))
            plt.ylabel('F{} ({}%)'.format(d2+1, round(100*pca.explained_variance_ratio_[d2],1)))

            plt.title("Projection des {} individus sur F{} et F{}".format(X_projected.shape[0], d1+1, d2+1), fontdict=font_title)
            plt.show(block=False)


plot_plans_factoriels_nutrigrade(X_projected, 4, pca, [(0,1),(2,3)], illustrative_var = df_nutriscore['nutrition_grade_fr'])


def plot_plans_factoriels(X_projected, n_comp, pca, axis_ranks, labels=None, alpha=1, illustrative_var=None):
    for d1,d2 in axis_ranks:
        if d2 < n_comp:
            # initialisation de la figure       
            fig = plt.figure(figsize=(12,8))
        
            # affichage des points
            if illustrative_var is None:
                plt.scatter(X_projected[:, d1], X_projected[:, d2], alpha=alpha)
            else:
                illustrative_var = np.array(illustrative_var)
                for value in np.unique(illustrative_var):
                    selected = np.where(illustrative_var == value)
                    plt.scatter(X_projected[selected, d1], X_projected[selected, d2], alpha=alpha, label=value)
                plt.legend()
                

            # affichage des labels des points
            if labels is not None:
                for i,(x,y) in enumerate(X_projected[:,[d1,d2]]):
                    plt.text(x, y, labels[i],
                              fontsize='14', ha='center',va='center') 
                    
            
            # détermination des limites du graphique
            boundary = np.max(np.abs(X_projected[:, [d1,d2]]))*1.1
            plt.xlim([-boundary,boundary])
            plt.ylim([-boundary,boundary])
        
            # affichage des lignes horizontales et verticales
            plt.plot([-100, 100], [0, 0], color='grey', ls='--')
            plt.plot([0, 0], [-100, 100], color='grey', ls='--')

            # nom des axes, avec le pourcentage d'inertie expliqué
            plt.xlabel('F{} ({}%)'.format(d1+1, round(100*pca.explained_variance_ratio_[d1],1)))
            plt.ylabel('F{} ({}%)'.format(d2+1, round(100*pca.explained_variance_ratio_[d2],1)))

            plt.title("Projection des {} individus sur F{} et F{}".format(X_projected.shape[0], d1+1, d2+1), fontdict=font_title)
            plt.show(block=False)


plot_plans_factoriels(X_projected, 4, pca, [(0,1),(2,3)], illustrative_var = df_nutriscore['nutrition_grade_fr'])


# Copy et préparation des données
df_cluster_all = df_nutriscore.copy()
df_cluster = df_nutriscore[cols_quant_cont]


# Scaler
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
scaler.fit(df_cluster)
df_cluster_scaled = scaler.transform(df_cluster)


# Méthode du coude pour trouver le nombre de clusters
from sklearn.cluster import KMeans
from yellowbrick.cluster import KElbowVisualizer

# Instantiate 
model = KMeans()
visualizer = KElbowVisualizer(model, k=(1, 11))

# Fit the data to the visualizer
visualizer.fit(df_cluster_scaled)    
plt.grid(False)
visualizer.show()

<AxesSubplot: title={'center': 'Distortion Score Elbow for KMeans Clustering'}, xlabel='k', ylabel='distortion score'>


# Avec k=3 le meilleur hyper-paramètre pour KMeans
from sklearn.cluster import KMeans

k_means = KMeans(n_clusters=3)
kmeans = k_means.fit(scaler.transform(df_cluster))
df_cluster_all['cluster'] = kmeans.labels_
df_cluster_all.head()


plt.figure(figsize=(7, 7))
plt.title("Les Clusters", size=16,weight='bold')
nb_par_var = df_cluster_all['cluster'].sort_values().value_counts()
nb_par_var = nb_par_var.loc[sorted(nb_par_var.index)]
explode = [0.1]
for i in range(len(nb_par_var) - 1):
    explode.append(0)
wedges, texts, autotexts = plt.pie(nb_par_var, labels=nb_par_var.index, autopct='%1.1f%%', 
                                   colors=['b', 'darkblue', 'steelblue'], textprops={
                                       'fontsize': 16, 'color': 'black', 'backgroundcolor': 'w'},
                                   explode=explode)
axes = plt.gca()
axes.legend(wedges,nb_par_var.index,loc='center right',fontsize=14,
            bbox_to_anchor=(1,0,0.5, 1))
plt.show()


plt.figure(figsize=(15, 8))
sns.countplot(x='cluster', hue='nutrition_grade_fr', data=df_cluster_all,
              palette=colors_nutri)

plt.legend(loc=1)
plt.ylabel('Nombre de produits', labelpad=20, fontsize=14)
plt.xticks(fontsize=14)
plt.title('Répartition des groupes de clusters par groupes Nutrigrade', fontsize=16)
plt.grid(False)
plt.show()


# Groupe 0
# Condition
mask0 = df_cluster_all['cluster'] == 0
# création d'un dataframe du cluster 0
df0 = df_cluster_all[mask0]['energy_100g']
# On sauvegarde le describe de ce df dans une variable
serie_0 = df0.describe()

# Groupe 1
mask1 = df_cluster_all['cluster'] == 1
df1 = df_cluster_all[mask1]['energy_100g']
serie_1 = df1.describe()
# Groupe 2
mask2 = df_cluster_all['cluster'] == 2
df2 = df_cluster_all[mask2]['energy_100g']
serie_2 = df2.describe()


# df_cluster_all des statistiques par groupe
# On crée une liste contenant nos 3 describe
liste_cluster = [serie_0.values, serie_1.values, serie_2.values]
# index
cols_cluster = serie_1.index
# Création df_cluster_all pour comparer les describe de nos 3 df de cluster
df_cluster_all_energy = pd.DataFrame(liste_cluster, columns=cols_cluster, index=['Groupe 0', 'Groupe 1', 'Groupe 2'])
# On affiche le résultat des groupes
df_cluster_all_energy

	code	creator	created_datetime	last_modified_datetime	product_name	brands	categories_fr	countries_fr	additives_n	additives_fr	nutrition_grade_fr	main_category_fr	energy_100g	fat_100g	saturated_fat_100g	carbohydrates_100g	sugars_100g	fiber_100g	proteins_100g	salt_100g	sodium_100g	nutrition_score_fr_100g
27358	0035826100696	usda-ndb-import	2017-03-09 15:42:55	2017-03-09 15:42:55	Trail Mix, Peanuts, Pretzels, Sesame Sticks, C...	Food Lion, Food Town Stores Inc.	inconnu	États-Unis	1.0	E375 - Acide nicotinique	d	inconnu	2092.0	30.00	5.00	43.33	3.33	6.7	16.67	1.77800	0.700	12.0
118239	0726984190543	usda-ndb-import	2017-03-09 13:21:37	2017-03-09 13:21:37	Marionberry Preserves	Haggen	inconnu	États-Unis	2.0	E440 - Pectines,E330 - Acide citrique	0	inconnu	1255.0	0.00	0.00	70.00	50.00	5.0	0.00	0.00000	0.000	11.0
16783	0021908503165	usda-ndb-import	2017-03-09 14:32:44	2017-03-09 14:32:44	Premium Organic Country Style Potatoes	Cascadian Farm Organic, Small Planet Foods Inc.	inconnu	États-Unis	0.0		a	inconnu	247.0	0.00	0.00	14.12	1.18	1.2	1.18	0.04572	0.018	-1.0
35762	0041250664741	usda-ndb-import	2017-03-09 11:07:12	2017-03-09 11:07:12	Hardwood Smoked Thick Sliced Bacon	Meijer	inconnu	États-Unis	3.0	E339iii - Phosphate de sodium tribasique,E316 ...	0	inconnu	2301.0	45.00	17.50	0.00	12.40	4.4	35.00	5.20700	2.050	24.2
5989	0011161032175	usda-ndb-import	2017-03-09 20:11:49	2017-03-09 20:11:49	Wieners	Shur Fine	inconnu	États-Unis	4.0	E325 - Lactate de sodium,E339 - Orthophosphate...	e	inconnu	1393.0	28.89	11.11	2.22	2.22	0.0	13.33	1.75006	0.689	21.0

	count	mean	std	min	25%	50%	75%	max
additives_n	240304.0	1.787436	2.464454	0.0	0.00000	1.00000	3.00000	31.000000
ingredients_from_palm_oil_n	240304.0	0.017582	0.133063	0.0	0.00000	0.00000	0.00000	2.000000
energy_100g	240304.0	1123.056095	786.012530	0.0	389.00000	1100.00000	1674.00000	3776.000000
fat_100g	240304.0	12.054291	16.700297	0.0	0.00000	4.60000	19.00000	100.000000
saturated_fat_100g	240304.0	4.614094	7.549919	0.0	0.00000	1.25000	6.67000	100.000000
carbohydrates_100g	240304.0	31.203502	28.476440	0.0	6.45000	20.00000	56.76000	100.000000
sugars_100g	240304.0	15.283422	20.688020	0.0	1.01000	5.10000	22.58000	100.000000
fiber_100g	240304.0	2.435193	4.211174	0.0	0.00000	1.02000	3.30000	100.000000
proteins_100g	240304.0	7.113039	8.106444	0.0	0.71000	4.88000	10.00000	100.000000
salt_100g	240304.0	1.592437	6.193162	0.0	0.06858	0.59182	1.37414	100.000000
sodium_100g	240304.0	0.626730	2.436071	0.0	0.02700	0.23300	0.54300	39.370079
nutrition_score_fr_100g	240304.0	8.959197	8.787331	-15.0	1.00000	9.00000	15.80000	40.000000

Desc	energy_100g	fat_100g	saturated_fat_100g	carbohydrates_100g	sugars_100g	fiber_100g	proteins_100g	salt_100g	sodium_100g
mean	1123.056095	12.054291	4.614094	31.203502	15.283422	2.435193	7.113039	1.592437	0.626730
median	1100.000000	4.600000	1.250000	20.000000	5.100000	1.020000	4.880000	0.591820	0.233000
var	617813.126622	278.898761	57.001032	810.904287	427.992372	17.733913	65.714153	38.355100	5.934415
std	786.010895	16.700262	7.549903	28.476381	20.687977	4.211165	8.106427	6.193149	2.436065
skew	0.425871	2.231646	3.502017	0.623817	1.725949	5.409043	2.126997	11.073758	11.093051
kurtosis	-0.446573	6.514148	22.325536	-0.954933	2.479089	58.608099	8.571377	142.319727	142.787027
mode	0 0.0	0 0.0	0 0.0	0 0.0	0 0.0	0 0.0	0 0.0	0 0.0	0 0.0
Min	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
Max	3776.000000	100.000000	100.000000	100.000000	100.000000	100.000000	100.000000	100.000000	39.370079

Desc	nutrition_score_fr_100g
mean	9.131191
median	10.000000
var	81.995724
std	9.055149
skew	0.116733
kurtosis	-1.017174
mode	0 0.0
Min	-15.000000
Max	40.000000

Plage	nb_données	%_données
(-15, -1]	33037	16.241980
(0, 2]	20121	9.892087
(3, 10]	35405	17.406160
(10, 18]	58204	28.614832
(19, 40]	32121	15.791647

Analyse : Concevez une application au service de la santé publique

Projet-3 : Analyse des données¶

IDEE APPLICATION¶

1. Chargement du jeu de données (dataset en anglais)¶

Analyse des dates de création et modification des produits¶

ANALYSE UNIVARIEE¶

Quantitatives continues¶

Variable quantitative discrete : Le nutriscore est-il bien répartit ?¶

'nutrition-score-fr_100g'¶

Les variables qualitalives¶

Variables qualitatives nominales¶

Qui sont les sources de ces données ?¶

Les marques sont_elles disponibles sur le marché français ?¶

Catégories¶

Les additifs à surveillées sont ils des informations dont nous pourrions disposer ?¶

Additifs ciblés¶

Variables qualitatives ordinales¶

Répartition des nutrition_grades et l'application¶

ANALYSE BIVARIEES¶

Les corrélations¶

Fonctions de test pour les analyses¶

Tester la normalité¶

Nutriscore | Nutrigrade¶

Nutrigrade | Protéines¶

Nutrigrade | Sodium¶

ANALYSE MULTIVARIEE¶

Regression linéaire multiple¶

Analyse en Composantes Principales (ACP)¶

Clustering¶

BILAN¶

	count	unique	top	freq	first	last
code	240304	240304	0000000004530	1	NaT	NaT
creator	240304	2477	usda-ndb-import	153975	NaT	NaT
created_datetime	240304	126364	2017-03-09 10:37:09	19	2012-01-31 14:43:58	2017-04-20 21:13:06
last_modified_datetime	240304	119721	2015-08-09 17:35:48	22	2012-04-08 08:12:35	2017-04-21 00:53:41
product_name	240304	186889	Extra Virgin Olive Oil	192	NaT	NaT
brands	240304	46265	inconnue	3145	NaT	NaT
categories_fr	240304	16277	inconnu	178919	NaT	NaT
countries_fr	240304	81	États-Unis	155671	NaT	NaT
additives_fr	240304	38480		102639	NaT	NaT
nutrition_grade_fr	240304	6	d	57653	NaT	NaT
main_category_fr	240304	2322	inconnu	178919	NaT	NaT

	Nom_brands	Nbr_brands	Fréquence (%)
0	Carrefour	2362	0.995956
1	Auchan	1768	0.745491
2	Meijer	1702	0.717662
3	U	1681	0.708807
4	Kroger	1454	0.613091
5	Leader Price	1372	0.578515
6	Casino	1233	0.519904
7	Ahold	1181	0.497978
8	Roundy's	1112	0.468884
9	Spartan	1063	0.448223

Nom_categories	Nbr_categories	Fréquence (%)
Snacks sucrés,Biscuits et gâteaux,Biscuits	691	1.125682
Snacks sucrés,Chocolats,Chocolats noirs	529	0.861774
Aliments et boissons à base de végétaux,Aliments d'origine végétale,Petit-déjeuners,Céréales et pommes de terre,Céréales et dérivés,Céréales pour petit-déjeuner	450	0.733078
Snacks sucrés,Biscuits et gâteaux,Biscuits,Biscuits au chocolat	409	0.666287
Snacks salés,Apéritif,Biscuits apéritifs	404	0.658141
Snacks sucrés,Confiseries,Bonbons	374	0.609269
Snacks sucrés,Chocolats,Chocolats au lait	363	0.591350
Produits laitiers,Yaourts	349	0.568543
Snacks sucrés,Chocolats	301	0.490348
Epicerie,Sauces	299	0.487090

Nom_Main categories	Nbr_Main categories	Fréquence (%)
Boissons	2290	3.730553
Epicerie	2278	3.711004
Aliments et boissons à base de végétaux	2274	3.704488
Chocolats	2252	3.668649
Conserves	2014	3.280932
Biscuits	1853	3.018653
Plats préparés	1813	2.953490
Surgelés	1713	2.790584
Petit-déjeuners	1566	2.551112
Snacks sucrés	1515	2.468030

Nom_Additives	Nbr_Additives	Fréquence (%)
E452vi - Tripolyphosphate de sodium et de potassium	345	1.564271
E339iii - Phosphate de sodium tribasique,E316 - Erythorbate de sodium,E250 - Nitrite de sodium	290	1.314895
E339iii - Phosphate de sodium tribasique	286	1.296758
E339 - Orthophosphates de sodium	211	0.956699
E339 - Orthophosphates de sodium,E316 - Erythorbate de sodium,E250 - Nitrite de sodium	194	0.879619
E450 - Sels métalliques de diphosphates	178	0.807073
E325 - Lactate de sodium,E339 - Orthophosphates de sodium,E262ii,E316 - Erythorbate de sodium,E250 - Nitrite de sodium	177	0.802539
E325 - Lactate de sodium,E339iii - Phosphate de sodium tribasique,E262ii,E316 - Erythorbate de sodium,E250 - Nitrite de sodium	175	0.793471
E375 - Acide nicotinique,E101 - Riboflavine,E450 - Sels métalliques de diphosphates	159	0.720925
E341iii - Phosphate de tricalcium	132	0.598504

Desc	a	b	c	d	e
mean	-3.426405	0.908934	6.356685	14.059927	21.967796
median	-3.000000	1.000000	6.000000	14.000000	22.000000
var	4.623705	0.748122	6.329803	5.342098	10.310623
std	2.150280	0.864940	2.515910	2.311298	3.211016
skew	-0.662412	-0.590996	0.039678	-0.089960	-0.117970
kurtosis	0.731734	3.785613	-1.413975	-0.253618	2.004625
mode	0 -1.0	0 0.0	0 3.0	0 14.0	0 20.0
Min	-15.000000	-10.000000	2.000000	6.000000	10.000000
Max	17.000000	2.000000	10.000000	18.000000	40.000000

Desc	proteins_100g
mean	7.782852
median	5.710000
var	64.586628
std	8.036581
skew	2.011552
kurtosis	7.844765
mode	0 0.0
Min	0.000000
Max	100.000000

Plage	nb_données	%_données
(0, 4]	57852	28.441779
(5, 9]	44923	22.085494
(10, 14]	19937	9.801627
(15, 19]	9812	4.823874
(20, 24]	8187	4.024975
(25, 29]	2984	1.467024
(30, 34]	985	0.484256
(35, 39]	409	0.201077
(40, 100]	850	0.417885

	a	b	c	d	e
count	31373.000000	31373.000000	31373.000000	31373.000000	31373.000000
mean	8.365304	5.109088	7.172807	8.226714	9.567869
std	7.051328	6.608959	7.564990	8.394596	9.348493
min	0.000000	0.000000	0.000000	0.000000	0.000000
25%	3.000000	0.620000	1.220000	2.500000	3.450000
50%	7.140000	3.200000	5.400000	6.060000	6.250000
75%	12.280000	7.300000	10.530000	12.390000	13.640000
max	100.000000	100.000000	93.330000	80.000000	100.000000

Plage	nb_données	%_données
(0.0, 0.2]	71320	35.063052
(0.3, 0.5]	37544	18.457757
(0.5, 0.7]	22252	10.939751
(0.8, 1.0]	8500	4.178855
(1.1, 1.5]	6109	3.003368
(1.6, 2.5]	3974	1.953738
(2.6, 3.0]	439	0.215826
(3.0, 4.0]	622	0.305794
(4.0, 40.0]	2248	1.105184

	a	b	c	d	e
count	32971.000000	31373.000000	42096.000000	57653.000000	39312.000000
mean	0.130486	0.207218	0.672157	0.647881	0.592513
std	0.155508	0.279822	2.652204	1.489795	1.185153
min	0.000000	0.000000	0.000000	0.000000	0.000000
25%	0.006000	0.017000	0.050000	0.071000	0.100000
50%	0.051181	0.167000	0.349303	0.366142	0.357000
75%	0.240079	0.365000	0.576000	0.724000	0.740000
max	4.666600	30.708661	39.370079	35.710000	39.370079

	feature	CORR_F1	CORR_F2	CORR_F3	CORR_F4
0	energy_100g	0.919577	-0.055475	0.003847	0.212015
1	fat_100g	0.777394	0.143944	-0.411599	-0.055548
2	saturated_fat_100g	0.752975	0.107812	-0.317509	-0.285934
3	carbohydrates_100g	0.413601	-0.334804	0.665704	0.353578
4	sugars_100g	0.436849	-0.368174	0.683280	-0.136313
5	fiber_100g	0.182201	-0.068756	-0.036149	0.861720
6	proteins_100g	0.252664	0.264638	-0.510740	0.429292
7	salt_100g	0.001630	0.926159	0.369268	0.057356
8	sodium_100g	0.001721	0.926165	0.369262	0.057372
9	nutrition_score_fr_100g	0.830060	0.120236	0.139848	-0.347973

	count	mean	std	25%	50%	75%	max
Groupe 0	98218.0	535.007854	374.108366	222.0	448.0	815.0	2134.0
Groupe 1	75141.0	1682.905476	385.920792	1464.0	1655.0	1971.0	3736.0
Groupe 2	30046.0	2004.213482	598.852161	1494.0	1950.0	2389.0	3776.0

	code	creator	created_datetime	last_modified_datetime	product_name	brands	categories_fr	countries_fr	additives_n	additives_fr	nutrition_grade_fr	main_category_fr	energy_100g	fat_100g	saturated_fat_100g	carbohydrates_100g	sugars_100g	fiber_100g	proteins_100g	salt_100g	sodium_100g	nutrition_score_fr_100g	cluster
0	0000000004530	usda-ndb-import	2017-03-09 14:32:37	2017-03-09 14:32:37	Banana Chips Sweetened (Whole)	inconnue	inconnu	États-Unis	0.0		d	inconnu	2243.0	28.57	28.57	64.29	14.29	3.6	3.57	0.00000	0.000	14.0	1
1	0000000004559	usda-ndb-import	2017-03-09 14:32:37	2017-03-09 14:32:37	Peanuts	Torn & Glasser	inconnu	États-Unis	0.0		b	inconnu	1941.0	17.86	0.00	60.71	17.86	7.1	17.86	0.63500	0.250	0.0	1
2	0000000016087	usda-ndb-import	2017-03-09 10:35:31	2017-03-09 10:35:31	Organic Salted Nut Mix	Grizzlies	inconnu	États-Unis	0.0		d	inconnu	2540.0	57.14	5.36	17.86	3.57	7.1	17.86	1.22428	0.482	12.0	2
6	0000000016124	usda-ndb-import	2017-03-09 10:35:11	2017-03-09 10:35:12	Organic Muesli	Daddy's Muesli	inconnu	États-Unis	2.0	E123 - Amarante,E307a - Tocophérol	c	inconnu	1833.0	18.75	4.69	57.81	15.62	9.4	14.06	0.13970	0.055	7.0	1
10	0000000016872	usda-ndb-import	2017-03-09 10:34:10	2017-03-09 10:34:11	Zen Party Mix	Sunridge	inconnu	États-Unis	1.0	E100 - Curcumine	d	inconnu	2230.0	36.67	5.00	36.67	3.33	6.7	16.67	1.60782	0.633	12.0	2